Compare Pastes
Differences between the pastes
#128398 (21.10.2019 18:01)
and
#220909 (17.09.2021 10:34).
| 1 | input {
| |
| 2 | beats {
| |
| 3 | port => 5044 | |
| 4 | } | |
| 5 | } | |
| 6 | filter {
| |
| 7 | grok {
| |
| 8 | match => { "message" => "%{SYSLOGTIMESTAMP} %{SYSLOGHOST} %{DATA:program}(?:\[%{POSINT}\])?: %{GREEDYDATA:message}" }
| |
| 9 | overwrite => "message" | |
| 10 | } | |
| 11 | mutate {
| |
| 12 | rename => ["host", "server"] | |
| 13 | convert => {"server" => "string"}
| |
| 14 | } | |
| 15 | ||
| 16 | ||
| 17 | } | |
| 18 | ||
| 19 | ||
| 20 | filter {
| |
| 21 | # grok log lines by program name (listed alpabetically) | |
| 22 | if [program] =~ /^postfix.*\/anvil$/ {
| |
| 23 | grok {
| |
| 24 | patterns_dir => "/etc/logstash/patterns" | |
| 25 | match => [ "message", "^%{POSTFIX_ANVIL}$" ]
| |
| 26 | tag_on_failure => [ "_grok_postfix_anvil_nomatch" ] | |
| 27 | add_tag => [ "_grok_postfix_success" ] | |
| 28 | } | |
| 29 | } else if [program] =~ /^postfix.*\/bounce$/ {
| |
| 30 | grok {
| |
| 31 | patterns_dir => "/etc/logstash/patterns" | |
| 32 | match => [ "message", "^%{POSTFIX_BOUNCE}$" ]
| |
| 33 | tag_on_failure => [ "_grok_postfix_bounce_nomatch" ] | |
| 34 | add_tag => [ "_grok_postfix_success" ] | |
| 35 | } | |
| 36 | } else if [program] =~ /^postfix.*\/cleanup$/ {
| |
| 37 | grok {
| |
| 38 | patterns_dir => "/etc/logstash/patterns" | |
| 39 | match => [ "message", "^%{POSTFIX_CLEANUP}$" ]
| |
| 40 | tag_on_failure => [ "_grok_postfix_cleanup_nomatch" ] | |
| 41 | add_tag => [ "_grok_postfix_success" ] | |
| 42 | } | |
| 43 | } else if [program] =~ /^postfix.*\/dnsblog$/ {
| |
| 44 | grok {
| |
| 45 | patterns_dir => "/etc/logstash/patterns" | |
| 46 | match => [ "message", "^%{POSTFIX_DNSBLOG}$" ]
| |
| 47 | tag_on_failure => [ "_grok_postfix_dnsblog_nomatch" ] | |
| 48 | add_tag => [ "_grok_postfix_success" ] | |
| 49 | } | |
| 50 | } else if [program] =~ /^postfix.*\/error$/ {
| |
| 51 | grok {
| |
| 52 | patterns_dir => "/etc/logstash/patterns" | |
| 53 | match => [ "message", "^%{POSTFIX_ERROR}$" ]
| |
| 54 | tag_on_failure => [ "_grok_postfix_error_nomatch" ] | |
| 55 | add_tag => [ "_grok_postfix_success" ] | |
| 56 | } | |
| 57 | } else if [program] =~ /^postfix.*\/local$/ {
| |
| 58 | grok {
| |
| 59 | patterns_dir => "/etc/logstash/patterns" | |
| 60 | match => [ "message", "^%{POSTFIX_LOCAL}$" ]
| |
| 61 | tag_on_failure => [ "_grok_postfix_local_nomatch" ] | |
| 62 | add_tag => [ "_grok_postfix_success" ] | |
| 63 | } | |
| 64 | } else if [program] =~ /^postfix.*\/master$/ {
| |
| 65 | grok {
| |
| 66 | patterns_dir => "/etc/logstash/patterns" | |
| 67 | match => [ "message", "^%{POSTFIX_MASTER}$" ]
| |
| 68 | tag_on_failure => [ "_grok_postfix_master_nomatch" ] | |
| 69 | add_tag => [ "_grok_postfix_success" ] | |
| 70 | } | |
| 71 | } else if [program] =~ /^postfix.*\/pickup$/ {
| |
| 72 | grok {
| |
| 73 | patterns_dir => "/etc/logstash/patterns" | |
| 74 | match => [ "message", "^%{POSTFIX_PICKUP}$" ]
| |
| 75 | tag_on_failure => [ "_grok_postfix_pickup_nomatch" ] | |
| 76 | add_tag => [ "_grok_postfix_success" ] | |
| 77 | } | |
| 78 | } else if [program] =~ /^postfix.*\/pipe$/ {
| |
| 79 | grok {
| |
| 80 | patterns_dir => "/etc/logstash/patterns" | |
| 81 | match => [ "message", "^%{POSTFIX_PIPE}$" ]
| |
| 82 | tag_on_failure => [ "_grok_postfix_pipe_nomatch" ] | |
| 83 | add_tag => [ "_grok_postfix_success" ] | |
| 84 | } | |
| 85 | } else if [program] =~ /^postfix.*\/postdrop$/ {
| |
| 86 | grok {
| |
| 87 | patterns_dir => "/etc/logstash/patterns" | |
| 88 | match => [ "message", "^%{POSTFIX_POSTDROP}$" ]
| |
| 89 | tag_on_failure => [ "_grok_postfix_postdrop_nomatch" ] | |
| 90 | add_tag => [ "_grok_postfix_success" ] | |
| 91 | } | |
| 92 | } else if [program] =~ /^postfix.*\/postscreen$/ {
| |
| 93 | grok {
| |
| 94 | patterns_dir => "/etc/logstash/patterns" | |
| 95 | match => [ "message", "^%{POSTFIX_POSTSCREEN}$" ]
| |
| 96 | tag_on_failure => [ "_grok_postfix_postscreen_nomatch" ] | |
| 97 | add_tag => [ "_grok_postfix_success" ] | |
| 98 | } | |
| 99 | } else if [program] =~ /^postfix.*\/qmgr$/ {
| |
| 100 | grok {
| |
| 101 | patterns_dir => "/etc/logstash/patterns" | |
| 102 | match => [ "message", "^%{POSTFIX_QMGR}$" ]
| |
| 103 | tag_on_failure => [ "_grok_postfix_qmgr_nomatch" ] | |
| 104 | add_tag => [ "_grok_postfix_success" ] | |
| 105 | } | |
| 106 | } else if [program] =~ /^postfix.*\/scache$/ {
| |
| 107 | grok {
| |
| 108 | patterns_dir => "/etc/logstash/patterns" | |
| 109 | match => [ "message", "^%{POSTFIX_SCACHE}$" ]
| |
| 110 | tag_on_failure => [ "_grok_postfix_scache_nomatch" ] | |
| 111 | add_tag => [ "_grok_postfix_success" ] | |
| 112 | } | |
| 113 | } else if [program] =~ /^postfix.*\/sendmail$/ {
| |
| 114 | grok {
| |
| 115 | patterns_dir => "/etc/logstash/patterns" | |
| 116 | match => [ "message", "^%{POSTFIX_SENDMAIL}$" ]
| |
| 117 | tag_on_failure => [ "_grok_postfix_sendmail_nomatch" ] | |
| 118 | add_tag => [ "_grok_postfix_success" ] | |
| 119 | } | |
| 120 | } else if [program] =~ /^postfix.*\/smtp$/ {
| |
| 121 | grok {
| |
| 122 | patterns_dir => "/etc/logstash/patterns" | |
| 123 | match => [ "message", "^%{POSTFIX_SMTP}$" ]
| |
| 124 | tag_on_failure => [ "_grok_postfix_smtp_nomatch" ] | |
| 125 | add_tag => [ "_grok_postfix_success" ] | |
| 126 | } | |
| 127 | } else if [program] =~ /^postfix.*\/lmtp$/ {
| |
| 128 | grok {
| |
| 129 | patterns_dir => "/etc/logstash/patterns" | |
| 130 | match => [ "message", "^%{POSTFIX_LMTP}$" ]
| |
| 131 | tag_on_failure => [ "_grok_postfix_lmtp_nomatch" ] | |
| 132 | add_tag => [ "_grok_postfix_success" ] | |
| 133 | } | |
| 134 | } else if [program] =~ /^postfix.*\/smtpd$/ {
| |
| 135 | grok {
| |
| 136 | patterns_dir => "/etc/logstash/patterns" | |
| 137 | match => [ "message", "^%{POSTFIX_SMTPD}$" ]
| |
| 138 | tag_on_failure => [ "_grok_postfix_smtpd_nomatch" ] | |
| 139 | add_tag => [ "_grok_postfix_success" ] | |
| 140 | } | |
| 141 | } else if [program] =~ /^postfix.*\/postsuper$/ {
| |
| 142 | grok {
| |
| 143 | patterns_dir => "/etc/logstash/patterns" | |
| 144 | match => [ "message", "^%{POSTFIX_POSTSUPER}$" ]
| |
| 145 | tag_on_failure => [ "_grok_postfix_postsuper_nomatch" ] | |
| 146 | add_tag => [ "_grok_postfix_success" ] | |
| 147 | } | |
| 148 | } else if [program] =~ /^postfix.*\/tlsmgr$/ {
| |
| 149 | grok {
| |
| 150 | patterns_dir => "/etc/logstash/patterns" | |
| 151 | match => [ "message", "^%{POSTFIX_TLSMGR}$" ]
| |
| 152 | tag_on_failure => [ "_grok_postfix_tlsmgr_nomatch" ] | |
| 153 | add_tag => [ "_grok_postfix_success" ] | |
| 154 | } | |
| 155 | } else if [program] =~ /^postfix.*\/tlsproxy$/ {
| |
| 156 | grok {
| |
| 157 | patterns_dir => "/etc/logstash/patterns" | |
| 158 | match => [ "message", "^%{POSTFIX_TLSPROXY}$" ]
| |
| 159 | tag_on_failure => [ "_grok_postfix_tlsproxy_nomatch" ] | |
| 160 | add_tag => [ "_grok_postfix_success" ] | |
| 161 | } | |
| 162 | } else if [program] =~ /^postfix.*\/trivial-rewrite$/ {
| |
| 163 | grok {
| |
| 164 | patterns_dir => "/etc/logstash/patterns" | |
| 165 | match => [ "message", "^%{POSTFIX_TRIVIAL_REWRITE}$" ]
| |
| 166 | tag_on_failure => [ "_grok_postfix_trivial_rewrite_nomatch" ] | |
| 167 | add_tag => [ "_grok_postfix_success" ] | |
| 168 | } | |
| 169 | } else if [program] =~ /^postfix.*\/discard$/ {
| |
| 170 | grok {
| |
| 171 | patterns_dir => "/etc/logstash/patterns" | |
| 172 | match => [ "message", "^%{POSTFIX_DISCARD}$" ]
| |
| 173 | tag_on_failure => [ "_grok_postfix_discard_nomatch" ] | |
| 174 | add_tag => [ "_grok_postfix_success" ] | |
| 175 | } | |
| 176 | } else if [program] =~ /^postfix.*\/virtual$/ {
| |
| 177 | grok {
| |
| 178 | patterns_dir => "/etc/logstash/patterns" | |
| 179 | match => [ "message", "^%{POSTFIX_VIRTUAL}$" ]
| |
| 180 | tag_on_failure => [ "_grok_postfix_virtual_nomatch" ] | |
| 181 | add_tag => [ "_grok_postfix_success" ] | |
| 182 | } | |
| 183 | } else if [program] =~ /^postfix.*/ {
| |
| 184 | mutate {
| |
| 185 | add_tag => [ "_grok_postfix_program_nomatch" ] | |
| 186 | } | |
| 187 | } | |
| 188 | ||
| 189 | # process key-value data if it exists | |
| 190 | if [postfix.keyvalue_data] {
| |
| 191 | kv {
| |
| 192 | source => "postfix.keyvalue_data" | |
| 193 | trim_value => "<>," | |
| 194 | prefix => "postfix." | |
| 195 | remove_field => [ "postfix_keyvalue_data" ] | |
| 196 | } | |
| 197 | ||
| 198 | # some post processing of key-value data | |
| 199 | if [postfix.client] {
| |
| 200 | grok {
| |
| 201 | patterns_dir => "/etc/logstash/patterns" | |
| 202 | match => ["postfix.client", "^%{POSTFIX_CLIENT_INFO}$"]
| |
| 203 | tag_on_failure => [ "_grok_kv_postfix_client_nomatch" ] | |
| 204 | remove_field => [ "postfix_client" ] | |
| 205 | } | |
| 206 | } | |
| 207 | if [postfix.relay] {
| |
| 208 | grok {
| |
| 209 | patterns_dir => "/etc/logstash/patterns" | |
| 210 | match => ["postfix.relay", "^%{POSTFIX_RELAY_INFO}$"]
| |
| 211 | tag_on_failure => [ "_grok_kv_postfix_relay_nomatch" ] | |
| 212 | remove_field => [ "postfix_relay" ] | |
| 213 | } | |
| 214 | } | |
| 215 | if [postfix.delays] {
| |
| 216 | grok {
| |
| 217 | patterns_dir => "/etc/logstash/patterns" | |
| 218 | match => ["postfix.delays", "^%{POSTFIX_DELAYS}$"]
| |
| 219 | tag_on_failure => [ "_grok_kv_postfix_delays_nomatch" ] | |
| 220 | remove_field => [ "postfix_delays" ] | |
| 221 | } | |
| 222 | } | |
| 223 | } | |
| 224 | ||
| 225 | # process command counter data if it exists | |
| 226 | if [postfix.command_counter_data] {
| |
| 227 | grok {
| |
| 228 | patterns_dir => "/etc/logstash/patterns" | |
| 229 | match => ["postfix_command_counter_data", "^%{POSTFIX_COMMAND_COUNTER_DATA}$"]
| |
| 230 | tag_on_failure => ["_grok_postfix_command_counter_data_nomatch"] | |
| 231 | remove_field => ["postfix_command_counter_data"] | |
| 232 | } | |
| 233 | } | |
| 234 | ||
| 235 | # Do some data type conversions | |
| 236 | mutate {
| |
| 237 | convert => [ | |
| 238 | # list of integer fields | |
| 239 | "postfix.anvil_cache_size", "integer", | |
| 240 | "postfix.anvil_conn_count", "integer", | |
| 241 | "postfix.anvil_conn_rate", "integer", | |
| 242 | "postfix.client_port", "integer", | |
| 243 | "postfix.cmd_auth", "integer", | |
| 244 | "postfix.cmd_auth_accepted", "integer", | |
| 245 | "postfix.cmd_count", "integer", | |
| 246 | "postfix.cmd_count_accepted", "integer", | |
| 247 | "postfix.cmd_data", "integer", | |
| 248 | "postfix.cmd_data_accepted", "integer", | |
| 249 | "postfix.cmd_ehlo", "integer", | |
| 250 | "postfix.cmd_ehlo_accepted", "integer", | |
| 251 | "postfix.cmd_helo", "integer", | |
| 252 | "postfix.cmd_helo_accepted", "integer", | |
| 253 | "postfix.cmd_mail", "integer", | |
| 254 | "postfix.cmd_mail_accepted", "integer", | |
| 255 | "postfix.cmd_quit", "integer", | |
| 256 | "postfix.cmd_quit_accepted", "integer", | |
| 257 | "postfix.cmd_rcpt", "integer", | |
| 258 | "postfix.cmd_rcpt_accepted", "integer", | |
| 259 | "postfix.cmd_rset", "integer", | |
| 260 | "postfix.cmd_rset_accepted", "integer", | |
| 261 | "postfix.cmd_starttls", "integer", | |
| 262 | "postfix.cmd_starttls_accepted", "integer", | |
| 263 | "postfix.cmd_unknown", "integer", | |
| 264 | "postfix.cmd_unknown_accepted", "integer", | |
| 265 | "postfix.nrcpt", "integer", | |
| 266 | "postfix.postscreen_cache_dropped", "integer", | |
| 267 | "postfix.postscreen_cache_retained", "integer", | |
| 268 | "postfix.postscreen_dnsbl_rank", "integer", | |
| 269 | "postfix.relay_port", "integer", | |
| 270 | "postfix.server_port", "integer", | |
| 271 | "postfix.size", "integer", | |
| 272 | "postfix.status_code", "integer", | |
| 273 | "postfix.termination_signal", "integer", | |
| 274 | ||
| 275 | # list of float fields | |
| 276 | "postfix.delay", "float", | |
| 277 | "postfix.delay_before_qmgr", "float", | |
| 278 | "postfix.delay_conn_setup", "float", | |
| 279 | "postfix.delay_in_qmgr", "float", | |
| 280 | "postfix.delay_transmission", "float", | |
| 281 | "postfix.postscreen_violation_time", "float" | |
| 282 | ] | |
| 283 | } | |
| 284 | mutate {
| |
| 285 | rename => ["host", "server"] | |
| 286 | convert => {"server" => "string"}
| |
| 287 | } | |
| 288 | ||
| 289 | } | |
| 290 | ||
| 291 | output {
| |
| 292 | elasticsearch {
| |
| 293 | hosts => "localhost:9200" | |
| 294 | index => "postfix-%{+YYYY.MM.dd}"
| |
| 295 | } | |
| 296 | ||
| 297 | } | |
| 298 | ||
| 299 | ||
| 300 | ||
| 301 | ||
| 302 | ||
| 303 | ||
| 304 | ||
| 305 | ||
| 306 | ||
| 307 | ||
| 308 | ||
| 309 | ||
| 310 | ||
| 311 | ||
| 312 | ||
| 313 | ||
| 314 | ||
| 315 |
